# 异步爬虫
# 爬取梨视频的视频数据
import random
import re
from multiprocessing.dummy import Pool

import requests
from lxml import etree

if __name__ == '__main__':
    headers = {
        'User-Agent': 'Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.0.0 Mobile Safari/537.36',
        'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.0.0 Safari/537.36'
    }

    url = 'https://www.pearvideo.com/panorama'

    page_text = requests.get(url, headers).text
    etree_html = etree.HTML(page_text)
    # //表示查询所有子孙节点   /表示查询某个节点下的下一个节点,只查询子一辈 .表示选取当前节点
    li_list = etree_html.xpath('//ul[@id="listvideoListUl"]/li')

    # 真实视频地址列表
    video_urls = []
    for li in li_list:
        detail_url = 'https://www.pearvideo.com/' + li.xpath('./div/a/@href')[0]
        name = li.xpath('./div/a/div[2]/text()')[0] + '.mp4'
        # print(detail_url, name)
        # 获取视频id
        video_id = (li.xpath('./div/a/@href')[0]).split('_')[1]
        # print(video_id)

        # ajax请求
        ajax_url = 'https://www.pearvideo.com/videoStatus.jsp?'
        ajax_headers = {
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.128 Safari/537.36',
            'Referer': 'https://www.pearvideo.com/video_' + video_id
        }
        params = {
            'contId': video_id,
            'mrd': str(random.random())
        }
        page_dict = requests.get(url=ajax_url, params=params, headers=ajax_headers).json()
        # print(page_dict)
        '''
            响应示例：srcUrl是视频伪地址，真实地址：https://video.pearvideo.com/mp4/short/20240122/cont-{video_id}-71105261_adpkg-ad_hd.mp4
            如何知道视频真实地址？
                进入视频详情页，打开F12控制台->All，点击播放，出现"cont-1654170-15001098_adpkg-ad_hd.mp4"请求
            {
              "resultCode": "1",
              "resultMsg": "success",
              "reqId": "27e6d2f9-f150-4370-a988-1aee451032b9",
              "systemTime": "1705991215760",
              "videoInfo": {
                "playSta": "1",
                "video_image": "https://image.pearvideo.com/cont/20240122/cont-1791428-71069255.jpeg",
                "videos": {
                  "hdUrl": "",
                  "hdflvUrl": "",
                  "sdUrl": "",
                  "sdflvUrl": "",
                  "srcUrl": "https://video.pearvideo.com/mp4/short/20240122/1705991215760-71105261-hd.mp4"
                }
              }
            }
        '''
        # 解析视频伪地址
        fake_video_url = page_dict['videoInfo']['videos']['srcUrl']
        # print(fake_video_url)
        # 获取fake_video_url中的时间戳
        video_systemTime_dict = page_dict['systemTime']
        # 将fake_video_url中的时间戳替换为cont-video_id，得到真实地址
        video_url = re.sub(video_systemTime_dict, 'cont-' + video_id, fake_video_url)
        # print(video_url)

        video_dict = {'name': name, 'url': video_url}
        video_urls.append(video_dict)
print(video_urls)


def get_video(video_urls):
    download_url = video_urls['url']
    print('正在下载' + video_urls['name'] + '...')
    video_data = requests.get(download_url, headers).content

    # 持久化存储
    with open(video_urls['name'], 'wb') as fp:
        fp.write(video_data)
        print(video_urls['name'] + '下载完成！')


#  线程池爬取数据
pool = Pool(4)
pool.map(get_video, video_urls)
pool.close()
pool.join()
